require('colors')
const fs = require('fs')
const request = require('request-promise')
const cheerio = require('cheerio')
const moment = require('moment')
const md5 = require('../../lib/md5')
const redis = require('../../lib/redis')
const logger = require('../../config/log4j')

const siteUrl = 'https://shixian.com'
const entryUrl = `${siteUrl}/jobs`

let globalIndex = []
const idxFilename = `data/${'shixian-' + moment().format('YYYY-MM-DD')}.idx`

const getIndexData = filepath => {
  try {
    const content = fs.readFileSync(filepath, 'utf-8')

    const arr = content.split(',')
    return arr
  } catch (err) {}
  return []
}

const isAppend = rawKey => {
  const key = md5(rawKey).substring(0, 8)
  if (globalIndex.indexOf(key) !== -1) return false
  globalIndex.push(key)
  fs.appendFile(idxFilename, `${key},`, function() {
    console.log('创建索引成功'.green)
  })
  return true
}

const appendFile = item => {
  const filename = 'data/' + moment().format('YYYY-MM-DD') + '.json'
  if (isAppend(item.url)) {
    const itemStr = JSON.stringify(item)
    // 写入文件
    fs.appendFile(filename, itemStr + '\n', function() {
      console.log('消息写入文件成功'.green)
    })
    // 写入redis
    redis.queueIn(item)
  }
}

const filterItem = $ =>
  function(index, elem) {
    const reg = /.*分钟前发布|大约.*小时前发布/
    return reg.test(
      $(this)
        .find('.publish-at')
        .text()
    )
  }

const parseItem = $ =>
  function(index, elem) {
    const type = 'project'
    const title = $(this)
      .find('h5.title')
      .html()
      .trim()
    const desc = $(this)
      .find('.describe')
      .text()
      .trim()
    const price = $(this)
      .find('.about .price')
      .text()
      .trim()
    let url = $(this)
      .find('.info>a')
      .attr('href')
    url = `${siteUrl}${url}`
    const source = '实现网'
    const date = new Date()
    const item = {type, title, desc, price, url, source, date}
    appendFile(item)
  }

async function getJobInfo() {
  const resposne = await request.get({url: entryUrl}).catch(err => {
    console.log('【实现】request请求失败'.red)
    logger.error('【实现】request请求失败'.red)
    return ''
  })
  const $ = cheerio.load(resposne, {decodeEntities: false})
  $('.job-list')
    .children()
    .filter(filterItem($))
    .each(parseItem($))
}

async function shixianSpider() {
  globalIndex = getIndexData(idxFilename)
  await getJobInfo()
}

module.exports = shixianSpider
